package spider;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

import org.eweb4j.spiderman.fetcher.Page;
import org.eweb4j.spiderman.spider.SpiderListener;
import org.eweb4j.spiderman.spider.SpiderListenerAdaptor;
import org.eweb4j.spiderman.spider.Spiderman;
import org.eweb4j.spiderman.task.Task;
import org.eweb4j.spiderman.xml.Seed;
import org.eweb4j.spiderman.xml.Site;
import org.eweb4j.util.CommonUtil;
import org.eweb4j.util.FileUtil;
import org.htmlcleaner.XPatherException;

import net.zweb.core.SimpleZWeb;
import net.zweb.core.ZWeb;
import net.zweb.core.db.Db;

public class BaiduSpider {

	public static void main(String[] args) throws XPatherException {
		final ZWeb zweb = new SimpleZWeb();
		zweb.startup();
		final Db db = zweb.getDb();
		
		SpiderListener listener = new SpiderListenerAdaptor() {
			public void onStartup(Site site) {
				List<Seed> seeds = site.getSeeds().getSeed();
				// 输入百度关键词，动态增加种子链接
				for (String company : ENTERPRISES) {
					String encodedName = urlEncode("\""+company+"\"");
					//百度网页搜索
					Seed seed = new Seed();
					seed.setName(company);
					seed.setUrl("http://www.baidu.com/s?wd=" + encodedName);
					seeds.add(seed);
					
					// 百度新闻搜索
					Seed seed2 = new Seed();
					seed2.setName(company);
					seed2.setUrl("http://news.baidu.com/ns?word="+encodedName+"&pn=0&ct=1&tn=news&ie=utf-8&bt=0&et=0");
					seeds.add(seed2);
					
					// 百度知道搜索
					Seed seed3 = new Seed();
					seed3.setName(company);
					seed3.setUrl("http://zhidao.baidu.com/search?word="+encodedName+"&ie=utf-8&site=-1&sites=0&date=0&pn=0");
					seeds.add(seed3);
				}
			}
			
			public void onParse(Thread thread, Task task, List<Map<String, Object>> models) {
				Page page = task.page;
				
				for (Map<String, Object> model : models) {
					if (db.exists(DownloadModel.class, "url = ?", task.url)) {
						continue;
					}
					String company = task.seed.getName();
					String charset = page.getCharset();
					String title = (String)model.get("title");
					String url = task.url;
					String seedUrl = task.seed.getUrl();
					String pageContent = null;
					if ("zhidao".equals(task.target.getName())) {
						// 合并字段到page_content
						StringBuilder sb = new StringBuilder();
						sb.append(model.get("qTitle")).append("\r\n<br/>===\r\n<br/>");
						sb.append(model.get("qContent")).append("\r\n<br/>===\r\n<br/>");
						@SuppressWarnings("unchecked")
						List<String> list = (List<String>)model.get("lineContent");
						sb.append(CommonUtil.join(list, "\r\n<br/>##\r\n<br/>"));
						pageContent = sb.toString();
					} else {
						pageContent = (String)model.get("pageContent");
					}
					if (CommonUtil.isBlank(pageContent)) continue;
					
					// 是否符合关键词匹配 TODO 换成其它情感分析处理
					boolean save = KEYWORDS.isEmpty();
					List<String> ws = new ArrayList<String>();
					for (String kw : KEYWORDS) {
						if (pageContent.matches("[\\s\\S]*"+kw+"[\\s\\S]*")){
							ws.add(kw);
						}
					}
					if (!save) {
						save = !ws.isEmpty();
					}
					if (save) {
						// save to db
						DownloadModel values = new DownloadModel();
						values.setTitle(title);
						values.setUrl(url);
						values.setContent(pageContent);
						values.setCharset(charset);
						values.setCompany(company);
						values.setSeed_url(seedUrl);
						
						try {
							db.create(values);
						} catch (Throwable e) {
							e.printStackTrace();
							this.onError(thread, task, "保存到数据库失败:"+e.toString(), e);
						}
					}
				}
			}
		};

		Spiderman.me()
			.listen(listener)
			.init(new File("src/main/resources/baidu.xml"))
			.startup()
			.keepStrict("60m");
		
		zweb.shutdown();
	}
	
	static List<String> KEYWORDS = new ArrayList<String>();
	static {
		List<String> keywords = FileUtil.readLine(new File("dicts/keywords.txt"));
		if (keywords != null) {
			for (String kw : keywords) {
				KEYWORDS.addAll(Arrays.asList(kw.split(" ")));
			}
		}
	}
	
	static List<String> ENTERPRISES = new ArrayList<String>();
	static {
		List<String> ents = FileUtil.readLine(new File("lib/enterprises.txt"));
		if (ents != null) {
			for (int i = 0; i < ents.size(); i++) {
				String ent = ents.get(i);
				String name = ent.trim();
				if (name.endsWith("公司")) {
					ENTERPRISES.add(name);
				}
			}
		}
	}
	
	private static String urlEncode(String name) {
		try {
			return URLEncoder.encode(name, "UTF-8");
		} catch (UnsupportedEncodingException e) {
			return name;
		}
	}
	
	public static void writeFile(File f, String content, String charsetName) throws Exception {
		BufferedWriter writer = null;
		try {
			OutputStreamWriter osw = null;
			Charset c = null;
			try {
				c = Charset.forName(charsetName);
			} catch (Throwable e) {}
			
			if (c != null)
				osw = new OutputStreamWriter(new FileOutputStream(f), c.name());
			else 
				osw = new OutputStreamWriter(new FileOutputStream(f));
			
			writer = new BufferedWriter(osw);
			writer.write(content);
		} catch (Exception e) {
			throw e;
		} finally {
			if (writer != null) {
				try {
					writer.flush();
					writer.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}


}
